#!/usr/bin/env ruby

require 'set'

def load_file(file_path)
    items = []
    
    File.open(file_path) {|file|
        header = true
        columns = []
        file.each {|line| 
            substrs = line.chomp.split("\t")
            columns = substrs if header
            if header then header = false; next; end 
            
            item = {}
            (0..substrs.length-1).each {|i|
               item[columns[i]] = substrs[i]
            }
            #puts "item=#{item.inspect}"
            items.push(item)
        }
    }
    
    return items
end

def extract_words(text, storage, mode)
    hash = {}
    text.scan(/\w{3,}/){ |m| 
        key = m.to_s.downcase
        if mode == 1 then 
            count = storage[key]
            count = 0 if count.nil?
            storage[key] = count + 1
        else
            hash[key] = 1
        end
    }
    
    hash.each { |k,v| 
        v0 = storage[k]
        v0 = 0 if v0.nil?
        storage[k] = v0 + v
    }
end



# entry point
#base = '/home/michael/data/_docs/michael/_projects/mkuz-samples/trunk/ruby/appstore/'
#items_grossing = load_file(base + 'Top Grossing iPhone Weather Apps_2011-10-18.cvs')
#items_free = load_file(base + 'Top Free iPhone Weather Apps_2011-10-18.cvs')
#items_paid = load_file(base + 'Top Paid iPhone Weather Apps_2011-10-18.cvs')

###
=begin
set_grossing = Set.new
items_grossing.each {|item| set_grossing << item['id'] }

set_free = Set.new
items_free.each {|item| set_free << item['id'] }

set_paid = Set.new
items_paid.each {|item| set_paid << item['id'] }

puts "free intersect paid = #{(set_free.intersection set_paid).inspect}"
puts "grossing - (free union paid) = #{(set_grossing - (set_free + set_paid)).inspect}"
puts "grossing intersect free = #{(set_grossing.intersection(set_free)).inspect}"
puts "grossing intersect paid = #{(set_grossing.intersection(set_paid)).inspect}"

set_paid_left = set_paid - set_grossing
set_free_left = set_free - set_grossing
###

arr_free = []
items_free.each { |item| arr_free << set_free_left.include?(item['id']) }
puts "arr_free=#{arr_free.inspect}"

arr_paid = []
items_paid.each { |item| arr_paid << set_paid_left.include?(item['id']) }
puts "arr_paid=#{arr_paid.inspect}"
=end

#####################

def save_items (file_path, items, word_columns)
    puts "writing to file #{file_path}..."
    File.open(file_path, 'w') {|f| 
        items_str = "index\tname\tfull_desc_length"
        word_columns.each {|col| items_str += "\t#{col}" }
        items_str += "\n"
        
        items.each { |item|
            item_str = ""
            item_str += "#{item['index']}"
            item_str += "\t#{item['name']}"
            item_str += "\t#{item['full_desc_length']}"
            puts "item_str=#{item_str}"
            puts "item=#{item.inspect}"
            item['words'].each { |count| item_str += "\t#{count}" }
            items_str += "#{item_str}\n"
        }
        
        f.write(items_str) 
    }
    puts "ok"
end


def analyze_items (items, word_columns)
    main_storage = {}
    i = 0
    items.each do |item|
        break if (i+=1) > 25 
        extract_words(item["full_desc"], main_storage, 2)
    end
    #puts "main_storage=#{main_storage.inspect}"

    #sort by count
    sorted_words = main_storage.sort_by { |k,v| v}
    sorted_words = sorted_words.reverse[0..500]
    #puts "sorted_words=#{sorted_words.inspect}"
    
    sorted_words.each { |pair| word_columns.push(pair[0]) }
    #puts "word_colums=#{word_columns.inspect}"
    
    items.each do |item|
        storage = {}
        extract_words(item["full_desc"], storage, 2)
        
        item_words = []
        (0..word_columns.length-1).each { |i| 
            count = storage[word_columns[i]]
            count = 0 if count.nil?
            item_words[i] = count
        }
        item['words'] = item_words
    end
    
end

file_path = 'Top Free iPhone Weather Apps_2011-10-17.cvs'
items = load_file(file_path)

word_columns = []
analyze_items(items, word_columns)
save_items("analyzed_" + file_path, items, word_columns)